Bevan Joubert¶
Project 7 - Computer Vision¶
In [1]:
# Import the required packages.
# Standard Packages
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import math
# Data preparation.
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
# Image Processing
from skimage.color import rgb2gray
from skimage.filters import gaussian
# Trnsorflow and Keras
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, MaxPooling2D, GlobalMaxPooling2D, Flatten, Dense, Dropout, BatchNormalization
from tensorflow.keras import optimizers
# Performance Metrics
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
# Do some formatting of the page.
from IPython.core.interactiveshell import InteractiveShell
pd.options.display.float_format = '{:,.2f}'.format
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:90% !important; }</style>"))
pd.set_option('display.max_columns', None)
pd.set_option('display.float_format', '{:0.4f}'.format)
In [2]:
# If running locally with GPU. Do this to prevent errors due to GPU memory limit.
physical_devices = tf.config.experimental.list_physical_devices('GPU')
if len(physical_devices) > 0:
tf.config.experimental.set_memory_growth(physical_devices[0], True)
1 Import and Visualize Data¶
In [3]:
# Import the data.
images = np.load("images.npy")
labels = pd.read_csv("labels.csv")
In [4]:
# View the shape of the images array.
images.shape
Out[4]:
In [5]:
images[0, 0, 0]
Out[5]:
In [6]:
# View the shape of the labels array.
labels.shape
Out[6]:
In [7]:
labels.info()
In [8]:
labels.nunique()
Out[8]:
In [9]:
# The labels are grouped by Label in the csv file, get the positions where each new group begins.
# This will allow us to view information from each group, in both the labels and images arrays, more easily.
def getClassPositions(classList):
classPositions = []
classPositions.append(0)
currentClass = classList[0]
for i in range(len(classList)):
if(currentClass != classList[i]):
classPositions.append(i)
currentClass = classList[i]
return classPositions
classPositions = getClassPositions(labels['Label'])
classPositions
Out[9]:
In [10]:
# Display the labels.
labels.iloc[classPositions]
Out[10]:
In [11]:
# Convert the labels dataframe to a list,it will be easier to handle.
labelsList = list(labels['Label'])
In [12]:
# Print the first image from every class.
def printImages(imageArray, labelArray, positions, colorMap=None, picSize=5):
''' Prints the images at "positions" in rows of 6 images '''
totalImages = len(positions)
imageCount = 0
rows = math.ceil(totalImages/6)
lastRowCols = totalImages%6
f, ax = plt.subplots(rows, 6, figsize=(25,(rows*picSize)))
if (rows>1):
for i in range(rows):
for j in range(6):
ax[i,j].imshow(imageArray[positions[imageCount]], colorMap)
ax[i,j].title.set_text(labelArray[positions[imageCount]])
imageCount = imageCount + 1
if(imageCount == (totalImages)):
return
else:
for i in range(6):
ax[i].imshow(imageArray[positions[imageCount]], colorMap)
ax[i].title.set_text(labelArray[positions[imageCount]])
imageCount = imageCount + 1
if(imageCount == (totalImages)):
return
printImages(images, labelsList, classPositions)
In [13]:
# If you want to see more images of each class.
# Print the first 5 images from every class.
'''
for i, pos in enumerate(classPositions):
f, ax = plt.subplots(1, 5, figsize=(25,4))
f.suptitle(labels['Label'][pos], fontsize=12)
for j in range(5):
ax[j].imshow(images[pos+j])
'''
Out[13]:
2 Data Preprocessing¶
In [14]:
# Scale the images from int of 0-255 to a float of 0-1.
images_rgb = images.copy()
images_rgb = images_rgb.astype('float32')
images_rgb = images_rgb/255
images_rgb[0,0,0]
Out[14]:
In [15]:
# Perform Gaussian blurring on the images.
for i in range(images_rgb.shape[0]):
images_rgb[i] = gaussian(images_rgb[i], sigma=1.0, multichannel=True)
In [16]:
# Print the gaussian blurred first image from every class.
printImages(images_rgb, labelsList, classPositions)
In [17]:
# Sometimes CNNs only need grayscale images, so create grayscale images as well.
# Convert the image array to gray scale.
images_gs = rgb2gray(images)
images_gs.shape
Out[17]:
In [18]:
# Print the first image from every class in grayscale.
printImages(images_gs, labelsList, classPositions, 'gray')
In [19]:
# The rgb2gray function converts the images from an int between 0-255 to a float between 0-1.
# Confirms this.
images_gs[0, 0, 0] # Show the first pixel value to confirm
Out[19]:
In [20]:
# Perform Gaussian blurring on the grayscale images.
for i in range(images_gs.shape[0]):
images_gs[i] = gaussian(images_gs[i], sigma=1.0, multichannel=True)
In [21]:
# Confirm that the pixel values are still floating point and between 0 and 1.
images_gs[0, 0, 0] # Show the first pixel value to confirm
Out[21]:
In [22]:
# Print the grayscale gaussian blurred first image from every class.
printImages(images_gs, labelsList, classPositions, 'gray')
In [23]:
# Data is still floating point with values between 0 and 1, so there is no need for normalization.
images_gs[0,0,0]
Out[23]:
3a Make Data Compatible (Grayscale)¶
In [23]:
# Display the labels.
labels.iloc[classPositions]
Out[23]:
In [24]:
# Make a series to store the class names as they are in the current order.
classNames = pd.Series(np.array(labels.iloc[classPositions]['Label']))
classNames
Out[24]:
In [25]:
# Rename the classes with alphabet letters in the order which they appear.
# This is not neccessary, but it will keep the columns in the same order as which the labels currently are when one hot encoding.
# Keeping the order will be handy later on when visualising predictions.
renameDict = {"Small-flowered Cranesbill":"A",
"Fat Hen":"B",
"Shepherds Purse":"C",
"Common wheat":"D",
"Common Chickweed":"E",
"Charlock":"F",
"Cleavers":"G",
"Scentless Mayweed":"H",
"Sugar beet":"I",
"Maize":"J",
"Black-grass":"K",
"Loose Silky-bent":"L",}
In [26]:
# First one-hot encode the labels.
labels_ohe = labels.replace(renameDict)
labels_ohe = pd.get_dummies(labels_ohe)
labels_ohe.iloc[classPositions]
Out[26]:
In [27]:
# The labels_ohe dataframe must be converted to an array.
labels_array = labels_ohe.to_numpy()
labels_array[classPositions]
Out[27]:
In [28]:
# Notic that the classes are in the same order as the labels, so printing label from each class makes an identity matrix.
# This is purely to aid visualization of predictions later on.
In [29]:
# Check the shape of the X data.
images_gs.shape
Out[29]:
In [30]:
# Split the data into training, validation sets.
X_train_gs, X_test_gs, y_train_gs, y_test_gs = train_test_split(images_gs, labels_array, test_size=0.3, random_state=9, stratify=labels_ohe)
In [31]:
# Splite the test data into validation and test datasets.
X_val_gs, X_test_gs, y_val_gs, y_test_gs = train_test_split(X_test_gs, y_test_gs, test_size=0.5, random_state=9, stratify=y_test_gs)
In [32]:
print("X_train shape:\t" + str(X_train_gs.shape))
print("y_train shape:\t" + str(y_train_gs.shape))
print("X_val shape:\t" + str(X_val_gs.shape))
print("y_val shape:\t" + str(y_val_gs.shape))
print("X_test shape:\t" + str(X_test_gs.shape))
print("y_test shape:\t" + str(y_test_gs.shape))
In [33]:
# The X data must be reshaped to (:, 128, 128, 1)
X_train_gs = X_train_gs.reshape(X_train_gs.shape[0], X_train_gs.shape[1], X_train_gs.shape[2], 1)
X_val_gs = X_val_gs.reshape(X_val_gs.shape[0], X_val_gs.shape[1], X_val_gs.shape[2], 1)
X_test_gs = X_test_gs.reshape(X_test_gs.shape[0], X_test_gs.shape[1], X_test_gs.shape[2], 1)
print("X_train shape:\t" + str(X_train_gs.shape))
print("X_val shape:\t" + str(X_val_gs.shape))
print("X_test shape:\t" + str(X_test_gs.shape))
In [34]:
# Confirm that the datatypes of the image arrays are floats.
print("X_train datatype:\t" + str(X_train_gs[0,0,0,0].dtype))
print("X_val datatype:\t\t" + str(X_val_gs[0,0,0,0].dtype))
print("X_test datatype:\t" + str(X_test_gs[0,0,0,0].dtype))
In [35]:
# Confirm that image array values are between 0 and 1.
print("X_train min and max:\t" + str(X_train_gs.min()) + " " + str(X_train_gs.max()))
print("X_val min and max:\t" + str(X_val_gs.min()) + " " + str(X_val_gs.max()))
print("X_test min and max:\t" + str(X_test_gs.min()) + " " + str(X_test_gs.max()))
4a Model Building (Grayscale)¶
In [36]:
# Build the model.
model_gs = Sequential()
model_gs.add(Conv2D(filters=32, kernel_size=3, activation="relu", input_shape=(128, 128, 1)))
model_gs.add(Conv2D(filters=32, kernel_size=3, activation="relu"))
model_gs.add(Flatten())
model_gs.add(Dense(128, activation="relu"))
model_gs.add(Dense(12, activation="softmax"))
In [37]:
# Compile the model.
model_gs.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
In [38]:
# View the model summary.
model_gs.summary()
In [39]:
# Train the model.
history_gs = model_gs.fit(X_train_gs, y_train_gs, batch_size=32, epochs=10, validation_data=(X_val_gs, y_val_gs), verbose=1)
In [40]:
# Plot training history for initial grayscale model.
def plotFitHistory(fitHistory):
f, ax = plt.subplots(1, 2, figsize=(12,5))
ax[0].plot(fitHistory.history['accuracy'], label='Train')
ax[0].plot(fitHistory.history['val_accuracy'], label='Validation')
ax[0].title.set_text("Accuracy")
ax[0].legend()
ax[1].plot(fitHistory.history['loss'], label='Train')
ax[1].plot(fitHistory.history['val_loss'], label='Validation')
ax[1].title.set_text("Loss")
ax[1].legend()
plotFitHistory(history_gs)
In [41]:
# Predict the classes based on the validation X data.
y_valPred_gs = model_gs.predict_classes(X_val_gs)
In [42]:
# Convert from one hot encoding to indices.
y_val_ind_gs = np.argmax(y_val_gs, axis=1)
y_val_ind_gs[0:5]
Out[42]:
In [43]:
# Get the loss and accuracy based on the validation data.
loss_gs, accuracy_gs = model_gs.evaluate(X_val_gs, y_val_gs, verbose=0)
print("Validation Accuracy:\t{:.4f}".format(accuracy_gs))
print("Validation Loss:\t{:.4f}".format(loss_gs))
In [83]:
# Get the classification report based on the validation data.
print("Initial Grayscale Model Validation Classification Report")
print(classification_report(y_val_ind_gs, y_valPred_gs))
In [62]:
# Plot the validation confusion matrix for the initial grayscale model.
cm_gs=confusion_matrix(y_val_ind_gs, y_valPred_gs)
sns.heatmap(cm_gs, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Initial Grayscale Model Validation Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
Out[62]:
Grayscale Model Observations.¶
- The results are very poor and all over the place.
- The confusion matrix has no indication of a diagonal line which would form with better accuracy.
- It can be seen that the model becomes more overfit as the epochs increase.
- There is never a point where the validation accuracy is good at all. So the poor prediction is not just due to over fitting.
- I think that the pebbles are creating too many edges and that the colour is needed to distinguish the leaves better.
3a Make Data Compatible (RGB)¶
In [46]:
# Y Data is already prepared.
# New X data must be made from the gaussian blurred colour images.
In [47]:
# Check the shape of the X data.
images_rgb.shape
Out[47]:
In [48]:
# Split the data into training, validation sets.
X_train_rgb, X_test_rgb, y_train_rgb, y_test_rgb = train_test_split(images_rgb, labels_array, test_size=0.3, random_state=9, stratify=labels_ohe)
In [49]:
# Split the test data into validation and test datasets.
X_val_rgb, X_test_rgb, y_val_rgb, y_test_rgb = train_test_split(X_test_rgb, y_test_rgb, test_size=0.5, random_state=9, stratify=y_test_rgb)
In [50]:
print("X_train shape:\t" + str(X_train_rgb.shape))
print("y_train shape:\t" + str(y_train_rgb.shape))
print("X_val shape:\t" + str(X_val_rgb.shape))
print("y_val shape:\t" + str(y_val_rgb.shape))
print("X_test shape:\t" + str(X_test_rgb.shape))
print("y_test shape:\t" + str(y_test_rgb.shape))
In [51]:
# Confirm that image array values are between 0 and 1.
print("X_train min and max:\t" + str(X_train_rgb.min()) + " " + str(X_train_rgb.max()))
print("X_val min and max:\t" + str(X_val_rgb.min()) + " " + str(X_val_rgb.max()))
print("X_test min and max:\t" + str(X_test_rgb.min()) + " " + str(X_test_rgb.max()))
4b Model Building (RGB)¶
In [52]:
# Build the model.
model_rgb = Sequential()
model_rgb.add(Conv2D(filters=32, kernel_size=3, activation="relu", input_shape=(128, 128, 3), data_format='channels_last'))
model_rgb.add(Conv2D(filters=32, kernel_size=3, activation="relu"))
model_rgb.add(Flatten())
model_rgb.add(Dense(128, activation="relu"))
model_rgb.add(Dense(12, activation="softmax"))
In [53]:
# Compile the model.
model_rgb.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
In [54]:
# View the model summary.
model_rgb.summary()
In [55]:
# Train the model.
history_rgb = model_rgb.fit(X_train_rgb, y_train_rgb, batch_size=32, epochs=10, validation_data=(X_val_rgb, y_val_rgb), verbose=1)
In [56]:
# Plot training history for initial RGB model.
plotFitHistory(history_rgb)
In [57]:
# Make predictions based on the validation X data.
y_valPred_rgb = model_rgb.predict_classes(X_val_rgb)
In [58]:
# Convert from one hot encoding to indices.
y_val_ind_rgb = np.argmax(y_val_rgb, axis=1)
y_val_ind_rgb[0:5]
Out[58]:
In [59]:
# Get the loss and the accuracy
loss_rgb, accuracy_rgb = model_rgb.evaluate(X_val_rgb, y_val_rgb, verbose=0)
print("Validation Accuracy:\t{:.4f}".format(accuracy_rgb))
print("Validation Loss:\t{:.4f}".format(loss_rgb))
In [82]:
print("Initial RGB Model Validation Classification Report")
print(classification_report(y_val_ind_rgb, y_valPred_rgb))
In [64]:
cm_rgb=confusion_matrix(y_val_ind_rgb, y_valPred_rgb)
sns.heatmap(cm_rgb, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Initial RGB Model Validation Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
Out[64]:
RGB Model Observations:¶
- The results are much better than the grayscale model.
- The validation accuracy seems to reach a maximum value and then drops as the model becomes overfit.
- The confusion matrix looks much more like it should, with the diagonal clearly visible.
5 Fit and Evaluate Model (Refined Model)¶
In [65]:
# Continue with the RGB model, add some more layers to try and reduce overfitting and to get good results.
# Building the model was an iterative process, I ommitted showing each iteration and instead only show the model which I ended up with.
model_ref = Sequential()
model_ref.add(Conv2D(filters=32, kernel_size=3, padding='same', activation="relu", input_shape=(128, 128, 3), data_format='channels_last'))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.3))
model_ref.add(Conv2D(filters=32, kernel_size=3, padding='same', activation="relu"))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.2))
model_ref.add(Conv2D(filters=64, kernel_size=3, padding='same', activation="relu"))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Conv2D(filters=64, kernel_size=3, padding='same', activation="relu"))
model_ref.add(MaxPooling2D(pool_size=(2, 2)))
model_ref.add(Dropout(rate=0.1))
model_ref.add(GlobalMaxPooling2D())
model_ref.add(Flatten())
model_ref.add(Dense(512, activation="relu"))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Dense(256, activation="relu"))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Dense(128, activation="relu"))
model_ref.add(Dropout(rate=0.1))
model_ref.add(Dense(12, activation="softmax"))
In [66]:
# Compile the model.
model_ref.compile(loss="categorical_crossentropy", metrics=["accuracy"], optimizer="adam")
# View the model summary.
model_ref.summary()
In [67]:
# Train the model.
history_ref = model_ref.fit(X_train_rgb, y_train_rgb, batch_size=32, epochs=30, validation_data=(X_val_rgb, y_val_rgb), verbose=1)
In [68]:
# Plot training history for refined model.
plotFitHistory(history_ref)
In [69]:
# Make predictions based on the validation X data.
y_valPred_ref = model_ref.predict_classes(X_val_rgb)
# Convert from one hot encoding to indices.
y_val_ind_ref = np.argmax(y_val_rgb, axis=1)
In [71]:
# Get the training loss and the accuracy
trainLoss_ref, trainAccuracy_ref = model_ref.evaluate(X_train_rgb, y_train_rgb, verbose=0)
print("Training Accuracy:\t{:.4f}".format(trainAccuracy_ref))
print("Training Loss:\t\t{:.4f}".format(trainLoss_ref))
In [73]:
# Get the validation loss and the accuracy
validationLoss_ref, validationAccuracy_ref = model_ref.evaluate(X_val_rgb, y_val_rgb, verbose=0)
print("Validation Accuracy:\t{:.4f}".format(validationAccuracy_ref))
print("Validation Loss:\t{:.4f}".format(validationLoss_ref))
In [74]:
print("Refined Model Validation Classification Report")
print(classification_report(y_val_ind_ref, y_valPred_ref))
In [75]:
cm_rgb=confusion_matrix(y_val_ind_ref, y_valPred_ref)
sns.heatmap(cm_rgb, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Refined Model Validation Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
Out[75]:
In [76]:
# There are numerous instances of class 10 incorrectly classified as class 11.
# There are also quite a few instances of class 11 incorrectly classified as class 10.
# If these can be corrected then that will increase the accuracy.
# Find these instances.
class10IncorrectAs11 = []
class11IncorrectAs10 = []
for i in range(len(y_valPred_ref)):
if((y_val_ind_ref[i] == 10) and (y_valPred_ref[i] == 11)):
class10IncorrectAs11.append(i)
elif ((y_val_ind_ref[i] == 11) and (y_valPred_ref[i] == 10)):
class11IncorrectAs10.append(i)
In [77]:
# print some pictures of class 11 to compare to the incorrect classifications of class 10.
print("Example images from ", classNames[11])
printImages(images_rgb, labelsList, np.array(range(classPositions[11],classPositions[11]+6, 1)), picSize=3)
In [78]:
# Print the incorrectly predicted images from class 10.
print("Images from ", classNames[10], " incorrectly classified as ", classNames[11])
printImages(X_val_rgb, y_val_ind_ref, class10IncorrectAs11, picSize=3)
In [79]:
# Print some pictures of class 10 to compare to the incorrect classifications of class 11.
print("Example images from ", classNames[10])
printImages(images_rgb, labelsList, np.array(range(classPositions[11],classPositions[11]+6, 1)), picSize=3)
In [80]:
# Print the incorrectly predicted images from class 11.
print("Images from ", classNames[11], " incorrectly classified as ", classNames[10])
printImages(X_val_rgb, y_val_ind_ref, class11IncorrectAs10, picSize=3)
In [79]:
# It is easy to see why these are two classes are easily confused with each other. Both are a thin blade lea plant.
# classifying these images into the correct class probably requires advance techniques or just better images.
Evaluate the model on the test data¶
In [84]:
# Make predictions based on the validation X data.
y_testPred_prob = model_ref.predict(X_test_rgb)
y_testPred_class = model_ref.predict_classes(X_test_rgb)
# Convert from one hot encoding to indices.
y_test_ind_ref = np.argmax(y_test_rgb, axis=1)
# Get the loss and the accuracy
testLoss_ref, testAccuracy_ref = model_ref.evaluate(X_test_rgb, y_test_rgb, verbose=0)
print("Test Accuracy:\t{:.4f}".format(testAccuracy_ref))
print("Test Loss:\t{:.4f}".format(testLoss_ref))
In [85]:
print("Refined Model Test Classification Report")
print(classification_report(y_test_ind_ref, y_testPred_class))
In [86]:
cmTest_rgb=confusion_matrix(y_test_ind_ref, y_testPred_class)
sns.heatmap(cmTest_rgb, annot=True, fmt='g', xticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] , yticklabels = [0,1,2,3,4,5,6,7,8,9,10,11] )
plt.title("Refined Model Test Confusion Matrix")
plt.ylabel('Actual')
plt.xlabel('Predicted')
Out[86]:
6 Visualize Some Predictions¶
In [94]:
def visualizePrediction(index):
actualClass = y_test_ind_ref[index]
predictedClass = y_testPred_class[index]
print("Actual Class: ", actualClass)
print("Actual Name: "+classNames[actualClass])
plt.imshow(X_test_rgb[index])
plt.show()
print()
print("Predicted Class: ", predictedClass)
print("Predicted Name: "+classNames[predictedClass])
print("Softmax Probability Prediction: ")
print(y_testPred_prob[index])
if(actualClass != predictedClass):
print("Example of correct class")
plt.imshow(images_rgb[classPositions[predictedClass]])
plt.show()
print()
if(actualClass == predictedClass):
print("Correct Prediction")
else:
print("Incorrect Prediction")
print()
In [95]:
visualizePrediction(2)
In [96]:
visualizePrediction(3)
In [97]:
visualizePrediction(33)
In [98]:
visualizePrediction(36)
In [99]:
visualizePrediction(59)
In [100]:
# Visualize some incorrect predictions.
maxCount = 5
count = 0
for i in range(len(y_testPred_class)):
if(y_testPred_class[i] != y_test_ind_ref[i]):
visualizePrediction(i)
count += 1
if(count == maxCount):
break
Summary¶
In [101]:
trainDict = {'Loss': trainLoss_ref, 'Accuracy': trainAccuracy_ref}
trainSeries = pd.Series(name='Training', data=trainDict)
valDict = {'Loss': validationLoss_ref, 'Accuracy': validationAccuracy_ref}
valSeries = pd.Series(name='Validation', data=valDict)
testDict = {'Loss': testLoss_ref, 'Accuracy': testAccuracy_ref}
testSeries = pd.Series(name='Testing', data=testDict)
summary_df = pd.DataFrame(columns=['Accuracy', 'Loss'])
summary_df = summary_df.append(trainSeries)
summary_df = summary_df.append(valSeries)
summary_df = summary_df.append(testSeries)
In [102]:
summary_df
Out[102]: